{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x = \"\"\"\n",
    "# translation/malay-english/translated-0.json\n",
    "# translation/malay-english/translated-100000.json\n",
    "# translation/malay-english/translated-200000.json\n",
    "# translation/malay-english/translated-300000.json\n",
    "# translation/malay-english/translated-400000.json\n",
    "# translation/malay-english/translated-500000.json\n",
    "# translation/malay-english/translated-600000.json\n",
    "# translation/malay-english/translated-700000.json\n",
    "# translation/malay-english/translated-800000.json\n",
    "# translation/malay-english/translated-900000.json\n",
    "# translation/malay-english/translated-1000000.json\n",
    "# translation/malay-english/translated-1100000.json\n",
    "# translation/malay-english/translated-1200000.json\n",
    "# translation/malay-english/translated-1300000.json\n",
    "# translation/malay-english/translated-1400000.json\n",
    "# translation/malay-english/translated-1500000.json\n",
    "# translation/malay-english/translated-1600000.json\n",
    "# translation/malay-english/translated-1700000.json\n",
    "# translation/malay-english/translated-1800000.json\n",
    "# translation/malay-english/translated-1900000.json\n",
    "# translation/malay-english/translated-2000000.json\n",
    "# translation/malay-english/translated-2100000.json\n",
    "# translation/malay-english/translated-2200000.json\n",
    "# translation/malay-english/translated-2300000.json\n",
    "# translation/malay-english/translated-2400000.json\n",
    "# translation/malay-english/translated-2500000.json\n",
    "# translation/malay-english/translated-2600000.json\n",
    "# translation/malay-english/translated-2700000.json\n",
    "# translation/malay-english/translated-2800000.json\n",
    "# translation/malay-english/translated-2900000.json\n",
    "# translation/malay-english/translated-3000000.json\n",
    "# translation/malay-english/translated-3100000.json\n",
    "# translation/malay-english/translated-3200000.json\n",
    "# translation/malay-english/translated-3300000.json\n",
    "# translation/malay-english/translated-3400000.json\n",
    "# translation/malay-english/translated-3500000.json\n",
    "# \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# url = 'https://f000.backblazeb2.com/file/malay-dataset/'\n",
    "\n",
    "# for row in x.split('\\n'):\n",
    "#     if not len(row):\n",
    "#         continue\n",
    "#     link = f'{url}{row}'\n",
    "#     os.system(f'wget {link}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/parliament/translated-trainset-parliament.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/gnome-ms-en.json X\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/kde4-ms-en.json X\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/opensubtitle-ms-en.json X\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/qed-ms-en.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/tanzil-ms-en.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/translation/opus/ubuntu-ms-en.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('translated-trainset-parliament.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected = ['PERTANYAAN-PERTANYAAN JAWAB LISAN', 'PENGGAL KEEMPAT', 'PUSAT JAGAAN BERDAFTAR',\n",
    "           'BILANGAN PUSAT JAGAAN', 'pewan']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50460\n"
     ]
    }
   ],
   "source": [
    "selected, reject = [], []\n",
    "for row in data:\n",
    "    if any([r.lower() in row[0].lower() for r in rejected]):\n",
    "        reject.append(row)\n",
    "        continue\n",
    "    s = row[0]\n",
    "    if (sum(c.isdigit() for c in s) / len(s)) > 0.15:\n",
    "        reject.append(row)\n",
    "        continue\n",
    "    if sum(c.isalpha() for c in s) == 0:\n",
    "        reject.append(row)\n",
    "        continue\n",
    "    selected.append(row)\n",
    "    \n",
    "print(len(selected))\n",
    "x_parliament, y_parliament = list(zip(*selected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "with open('ubuntu-ms-en.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "X, Y = list(zip(*data))\n",
    "X = list(X)\n",
    "Y = list(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('qed-ms-en.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "x, y = list(zip(*data))\n",
    "X.extend(x)\n",
    "Y.extend(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tanzil-ms-en.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "x, y = list(zip(*data))\n",
    "X.extend(x)\n",
    "Y.extend(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['translated-3200000.json',\n",
       " 'translated-700000.json',\n",
       " 'translated-2100000.json',\n",
       " 'translated-3300000.json',\n",
       " 'translated-2300000.json',\n",
       " 'translated-2000000.json',\n",
       " 'translated-600000.json',\n",
       " 'translated-900000.json',\n",
       " 'translated-1000000.json',\n",
       " 'translated-1100000.json',\n",
       " 'translated-1900000.json',\n",
       " 'translated-500000.json',\n",
       " 'translated-1400000.json',\n",
       " 'translated-1500000.json',\n",
       " 'translated-2600000.json',\n",
       " 'translated-200000.json',\n",
       " 'translated-2900000.json',\n",
       " 'translated-3400000.json',\n",
       " 'translated-3500000.json',\n",
       " 'translated-2800000.json',\n",
       " 'translated-300000.json',\n",
       " 'translated-2500000.json',\n",
       " 'translated-3100000.json',\n",
       " 'translated-1300000.json',\n",
       " 'translated-2400000.json',\n",
       " 'translated-100000.json',\n",
       " 'translated-1600000.json',\n",
       " 'translated-2700000.json',\n",
       " 'translated-0.json',\n",
       " 'translated-800000.json',\n",
       " 'translated-1800000.json',\n",
       " 'translated-2200000.json',\n",
       " 'translated-1200000.json',\n",
       " 'translated-1700000.json',\n",
       " 'translated-3000000.json',\n",
       " 'translated-400000.json']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "translated = glob('translated*0.json')\n",
    "translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in translated:\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "\n",
    "    x, y = list(zip(*data))\n",
    "    X.extend(x)\n",
    "    Y.extend(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3808696, 3808696)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X), len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3808696/3808696 [01:11<00:00, 53290.02it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from unidecode import unidecode\n",
    "import re\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string).replace('\\n', ' ').replace('\\t', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string\n",
    "\n",
    "filtered_X, filtered_Y = [], []\n",
    "\n",
    "for i in tqdm(range(len(X))):\n",
    "    X[i] = cleaning(X[i])\n",
    "    Y[i] = cleaning(Y[i])\n",
    "    if len(X[i]) and len(Y[i]):\n",
    "        filtered_X.append(X[i])\n",
    "        filtered_Y.append(Y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3808696, 3808565)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X), len(filtered_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Beliau pernah disenaraihitamkan sekitar 1950-an setelah beliau menyertai \"House Un-American Activities Committee\" yang juga menjadi Parti Komunis.',\n",
       "  'He was blacklisted around the 1950s after joining the House Un-American Activities Committee, also a Communist Party.'),\n",
       " ('Kemudian beliau mencuba pula membawa peranan menerusi \"Paper Dolls\" (1984) dan \"Capital News\" (1990), yang diterbitkan oleh American Broadcasting Company (ABC), dan sekali lagi menerusi \"Harts of the West\" (1993), untuk CBS.',\n",
       "  'He then tried to lead roles in \"Paper Dolls\" (1984) and \"Capital News\" (1990), published by the American Broadcasting Company (ABC), and again through \"Harts of the West\" (1993), for CBS.'),\n",
       " ('Beliau pernah berlakon dalam \"The Loner\" (yang berakhir dengan satu musim sahaja pada 1965 hingga 1966), dua rancangan TV yang gagal menerusi stesyen NBC iaitu \"San Francisco International Airport\" (1970/71) dan \"Joe Forrester\" (1975/76).',\n",
       "  'He starred in \"The Loner\" (which ended with only one season between 1965 and 1966), two failed TV shows through the NBC stations \"San Francisco International Airport\" (1970/71) and \"Joe Forrester\" (1975/76) ).'),\n",
       " ('Beliau memulakan kerjaya beliau semula apabila nama beliau dipulihkan oleh anggota FBI dan mula menumpukan kepada pembikinan drama TV.',\n",
       "  'He started his career again when his name was restored by FBI agents and began focusing on making TV dramas.'),\n",
       " ('Beliau seterusnya membintangi rancangan antologi eponim \"The Lloyd Bridges Show\", yang turut menampilkan kedua-dua orang anak lelakinya Beau Bridges dan Jeff Bridges.',\n",
       "  'She then starred in the anthology show \"The Lloyd Bridges Show,\" which also featured her two sons Beau Bridges and Jeff Bridges.'),\n",
       " ('dan Harriet Evelyn Brown.', 'and Harriet Evelyn Brown.'),\n",
       " ('Beliau pernah menuntut di Sekolah Tinggi (Senior) Eureka pada 1931.',\n",
       "  'He studied at Eureka High School in 1931.'),\n",
       " ('Bridges mula dikenali setelah memegang watak sebagai Mike Nelson, menerusi siri TV \"Sea Hunt\", (1958 - 1961).',\n",
       "  'Bridges first became known after playing the role of Mike Nelson, in the TV series \"Sea Hunt\", (1958 - 1961).'),\n",
       " ('Beliau juga pernah ditawarkan watak sebagai \"Captain Kirk\" sebelum watak tersebut berpindah kepada William Shatner.',\n",
       "  'He was also offered the character as \"Captain Kirk\" before the character was transferred to William Shatner.'),\n",
       " ('Di Pantai St. Pete.', 'At St. Pete.')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(filtered_X[-10:], filtered_Y[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(filtered_X, filtered_Y, test_size = 100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/translation/iium-confession/part1.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/translation/iium-confession/part2.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/translation/iium-confession/part3.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/translation/iium-confession/part4.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['part1.txt', 'part3.txt', 'part2.txt', 'part4.txt']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parts = glob('part*.txt')\n",
    "parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3990, 3990)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ls, rs = [], []\n",
    "\n",
    "for part in parts:\n",
    "    with open(part) as fopen:\n",
    "        data = fopen.read().split('\\n')\n",
    "\n",
    "    for row in data:\n",
    "        if not len(row.strip()):\n",
    "            continue\n",
    "        splitted = row.split('<>')\n",
    "        if len(splitted) != 2:\n",
    "            continue\n",
    "        l, r = splitted\n",
    "        l = cleaning(l)\n",
    "        r = cleaning(r)\n",
    "        if len(l) and len(r):\n",
    "            ls.append(l)\n",
    "            rs.append(r)\n",
    "    \n",
    "len(ls), len(rs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X.extend(ls)\n",
    "train_Y.extend(rs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Tp of coz la aku tak tunjuk.', \"But of course I didn't show it.\"),\n",
       " ('Lagi2 plak ramai awek2 cun kat sini kan.',\n",
       "  'There are beautiful girl here.'),\n",
       " ('Sepanjang 1st year x banyak pon bnda yg berlaku.',\n",
       "  'During the 1st year not many thing happened.'),\n",
       " ('Geram btol.', 'I was so angry.'),\n",
       " ('Bkn kebencian yg meluap2 mcm sebelom ni.',\n",
       "  \"It's not hatred that used to come to me.\"),\n",
       " ('Takkanla aku nk sangkut ngan si badak ni.',\n",
       "  \"Don't tell me I've fallen to this badak.\"),\n",
       " ('Tapi tu la.', \"But that's it.\"),\n",
       " ('Hilang la macho aku.', 'I will lost my macho.'),\n",
       " ('Aku buat hal aku.', 'I did my thing.'),\n",
       " ('Dia buat hal dia.', 'She did her thing.')]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(train_X[-10:], train_Y[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Menjelang 12 Oktober, CONAIE telah mengumumkan bahawa mereka akan mengambil bahagian dalam dialog dengan kerajaan Ecuador.',\n",
       "  'By October 12, CONAIE has announced that they will participate in a dialogue with the Ecuadorian government.'),\n",
       " ('Kalau Kami mahu, tentulah Kami akan turunkan kepada mereka satu mukjizat dari langit, yang menjadikan mereka tunduk kepadanya.',\n",
       "  'If We will, We can send down upon them a sign from heaven, at which their necks will stay bent in humility.'),\n",
       " ('Bendera Italia dan Uni Eropa KONTAN.',\n",
       "  'Italy and the European Union CONTACT flag.'),\n",
       " ('Raffy akhirnya dengan sengaja mengubah strateginya.',\n",
       "  'Raffy eventually deliberately changed his strategy.'),\n",
       " ('XXXIII, 27-267.', 'XXXIII, 27-267.'),\n",
       " ('SIK: Isu tiga tandak atau kolam takungan atas bukit dibina sebagai sumber bekalan air haram penduduk yang tercemar susulan pembalakan di Hutan Simpan Bukit Enggang, di sini, selesai dengan pembinaan tandak air baharu di Bukit Padang, di sini.',\n",
       "  'SIK: The issue of three trenches or hillside reservoir was built as a source of contaminated water supply by residents following logging at the Enggang Bukit Forest Reserve, here, complete with the construction of a new waterway at Bukit Padang, here.'),\n",
       " ('Hanya selepas perintah mahkamah ini dibuat, wang itu akan dikreditkan ke dalam akaun pemulihan aset 1MDB yang diasingkan, yang dibuka semata-mata untuk mendapatkan semula aset-aset 1MDB untuk membayar hutang 1MDB, \" katanya.',\n",
       "  'Only after this court order is made will the money be credited into the 1MDB segregated asset recovery account, which is opened solely to recover 1MDB\\'s assets to pay off 1MDB\\'s debt, \"he said.'),\n",
       " ('Looks like Javascript is disabled in your browser.',\n",
       "  'Looks like Javascript is disabled in your browser.'),\n",
       " ('Oleh itu, bertasbihlah dengan memuji nama Tuhanmu Yang Maha Besar.',\n",
       "  'So exalt the Name of your Lord, the Great.'),\n",
       " ('Lebih daripada itu akan dikenakan tindakan undang-undang.',\n",
       "  'More than that, it will be subject to legal action.')]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(test_X[-10:], test_Y[-10:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "train_X, train_Y = shuffle(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset.json', 'w') as fopen:\n",
    "    json.dump({'train_X': train_X, 'train_Y': train_Y, 'test_X': test_X, 'test_Y': test_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘train’: File exists\n",
      "mkdir: cannot create directory ‘test’: File exists\n"
     ]
    }
   ],
   "source": [
    "!mkdir train\n",
    "!mkdir test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train/left.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train_X))\n",
    "    \n",
    "with open('train/right.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train_Y))\n",
    "    \n",
    "with open('test/left.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(test_X))\n",
    "    \n",
    "with open('test/right.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(test_Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train/\n",
      "train/left.txt\n",
      "train/right.txt\n",
      "test/\n",
      "test/left.txt\n",
      "test/right.txt\n"
     ]
    }
   ],
   "source": [
    "!tar -czvf train-ms-en.tar.gz train\n",
    "!tar -czvf test-ms-en.tar.gz test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
